package crawler.web.engine.custom;

import java.net.UnknownHostException;

import javax.annotation.Resource;

import library.functions.Function2PR;
import library.utils.probability.ProbabilityUtils;

import org.springframework.stereotype.Component;

import crawler.web.beans.CrawlURL;
import crawler.web.beans.ProxySetting;
import crawler.web.engine.BDBFrontier;
import crawler.web.engine.config.SpBN;
import crawler.web.framework.Crawler;

/**
 * @author waynechen Feb 24, 2012 3:28:27 PM
 */
@Component(value = "CrawlFrequencyFilter")
public class CrawlFrequencyFilter implements Function2PR<Crawler, CrawlURL, Boolean> {

	private static int getRandom(int $int) {
		$int = ProbabilityUtils.RAN.nextInt() % $int + $int + $int / 3;
		return $int;
	}

	@Resource(name = "ProxyDistributor")
	private ProxyDistributor proxyDistributor;

	@Resource(name = SpBN.bean_hosts_frontier)
	private BDBFrontier hosts;

	@Resource(name = SpBN.conf_crawlIntervalSecondPerHost)
	private Integer interval;

	@Override
	public Boolean apply(Crawler crawler, CrawlURL url) {
		//获取参数
		CrawlURL hostsURL = null;
		ProxySetting ps = null;
		String ip;
		try {
			ip = url.getIp();
		} catch (UnknownHostException e) {
			//失败后, 增加失败次数
			url.setRetryTimes(url.getRetryTimes() + 1);
			return false;
		}
		if (proxyDistributor != null) {
			ps = proxyDistributor.getProxy(crawler);
			ip = ip + "#" + ps.getIp() + ":" + ps.getProt();
			url.setProxy(ps.getIp() + ":" + ps.getProt());
		}

		//从时间时隔上检查url是否可以抓取
		Object object = hosts.get(ip);
		//从来没有访问过这个ip
		if (object == null) {
			url.setLastCrawlTime();
			hosts.put(ip, url);
		}
		//访问过
		else {
			hostsURL = (CrawlURL) object;
			long time = System.currentTimeMillis() - hostsURL.getLastCrawlTime().longValue();
			//时间已经过
			if (time >= getRandom(interval) * 1000) {
				hostsURL.setLastCrawlTime();
				hosts.put(ip, hostsURL);
			}
			//时间没有 到
			else {
				return false;//不能抓取本ip
			}
		}
		return true;//可以抓取本ip
	}
}
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//